# This file is part of Bika LIMS
#
# Copyright 2011-2016 by it's authors.
# Some rights reserved. See LICENSE.txt, AUTHORS.txt.

""" Shimadzu's 'GCMS QP2010 SE'
"""
from DateTime import DateTime
from Products.Archetypes.event import ObjectInitializedEvent
from Products.CMFCore.utils import getToolByName
from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile
from bika.lims import bikaMessageFactory as _
from bika.lims.utils import t
from bika.lims import logger
from bika.lims.browser import BrowserView
from bika.lims.idserver import renameAfterCreation
from bika.lims.utils import changeWorkflowState
from bika.lims.utils import tmpID
from cStringIO import StringIO
from datetime import datetime
from operator import itemgetter
from plone.i18n.normalizer.interfaces import IIDNormalizer
from zope.component import getUtility
import csv
import json
import plone
import re
import zope
import zope.event
from bika.lims.exportimport.instruments.resultsimport import InstrumentCSVResultsFileParser,\
    AnalysisResultsImporter
import traceback

title = "Shimadzu - GCMS-QP2010 SE"


def Import(context, request):
    """ Read Shimadzu's GCMS-QP2010 SE analysis results
    """
    form = request.form
    #TODO form['file'] sometimes returns a list
    infile = form['file'][0] if isinstance(form['file'],list) else form['file']
    artoapply = form['artoapply']
    override = form['override']
    sample = form.get('sample', 'requestid')
    instrument = form.get('instrument', None)
    errors = []
    logs = []

    # Load the most suitable parser according to file extension/options/etc...
    parser = None
    if not hasattr(infile, 'filename'):
        errors.append(_("No file selected"))
    parser = GCMSQP2010SECSVParser(infile)

    if parser:
        # Load the importer
        status = ['sample_received', 'attachment_due', 'to_be_verified']
        if artoapply == 'received':
            status = ['sample_received']
        elif artoapply == 'received_tobeverified':
            status = ['sample_received', 'attachment_due', 'to_be_verified']

        over = [False, False]
        if override == 'nooverride':
            over = [False, False]
        elif override == 'override':
            over = [True, False]
        elif override == 'overrideempty':
            over = [True, True]

        sam = ['getRequestID', 'getSampleID', 'getClientSampleID']
        if sample =='requestid':
            sam = ['getRequestID']
        if sample == 'sampleid':
            sam = ['getSampleID']
        elif sample == 'clientsid':
            sam = ['getClientSampleID']
        elif sample == 'sample_clientsid':
            sam = ['getSampleID', 'getClientSampleID']

        importer = GCMSQP2010SEImporter(parser=parser,
                                           context=context,
                                           idsearchcriteria=sam,
                                           allowed_ar_states=status,
                                           allowed_analysis_states=None,
                                           override=over,
                                           instrument_uid=instrument)
        tbex = ''
        try:
            importer.process()
        except:
            tbex = traceback.format_exc()
        errors = importer.errors
        logs = importer.logs
        warns = importer.warns
        if tbex:
            errors.append(tbex)

    results = {'errors': errors, 'log': logs, 'warns': warns}

    return json.dumps(results)


class GCMSQP2010SECSVParser(InstrumentCSVResultsFileParser):

    HEADERTABLE_KEY = '[Header]'
    HEADERKEY_FILENAME = 'Data File Name'
    HEADERKEY_OUTPUTDATE = 'Output Date'
    HEADERKEY_OUTPUTTIME = 'Output Time'

    FILEINFORMATION_KEY = '[File Information]'
    FILEINFORMATIONKEY_TYPE = 'Data File'
    FILEINFORMATIONKEY_GENERATED = 'Generated'
    FILEINFORMATIONKEY_GENERATEDBY = 'Generated By'
    FILEINFORMATIONKEY_MODIFIED = 'Modified'
    FILEINFORMATIONKEY_MODIFIEDBY = 'Modified by'

    SAMPLEINFORMATIONTABLE_KEY = '[Sample Information]'
    SAMPLEINFORMATIONKEY_OPERATORNAME = 'Operator Name'
    SAMPLEINFORMATIONKEY_ANALYZED = 'Analyzed'
    SAMPLEINFORMATIONKEY_TYPE = 'Type'
    SAMPLEINFORMATIONKEY_LEVEL = 'Level'
    SAMPLEINFORMATIONKEY_SAMPLENAME = 'Sample Name'
    SAMPLEINFORMATIONKEY_SAMPLEID = 'Sample ID'
    SAMPLEINFORMATIONRESULT_SAMPLEAMOUNT = 'Sample Amount'
    SAMPLEINFORMATIONRESULT_DILUTIONFACTOR = 'Dilution Factor'

    ORIGINALFILESTABLE_KEY = '[Original Files]'
    ORIGINALFILESKEY_DATAFILE = 'Data File'
    ORIGINALFILESKEY_METHODFILE = 'Method File'
    ORIGINALFILESKEY_BATCHFILE = 'Batch File'
    ORIGINALFILESKEY_REPORTFORMATFILE = 'Report Format File'
    ORIGINALFILESKEY_TUNINGFILE = 'Tuning File'

    FILEDESCRIPTION_KEY = '[File Description]'

    QUANTITATIONRESULTS_KEY = '[MS Quantitative Results]'
    QUANTITATIONRESULTS_NUMBEROFIDS = '# of IDs'
    QUANTITATIONRESULTS_HEADER_ID_NUMBER = 'ID#'
    QUANTITATIONRESULTS_NUMERICHEADERS = ('2nd', '1st', 'Constant',)

    SIMILARITYSEARCHRESULTS_KEY = '[MS Similarity Search Results for Identified Results]'
    PEAK_TABLE_KEY = '[MC Peak Table]'

    SEPERATOR = '\t'

    def __init__(self, csv):
        InstrumentCSVResultsFileParser.__init__(self, csv)
        self._end_header = False
        self._end_sampleinformationtable = False 
        self._sampleinformationresults = [] 
        self._sampleinformation = []

        self._quantitationresultsheader = []
        self._sampleinformationheader = []
        self._numline = 0

    def getAttachmentFileType(self):
        return "Agilent's Masshunter Quant CSV"

    def _parseline(self, line):
        if self._end_header == False:
            return self.parse_headerline(line)
        elif self._end_sampleinformationtable == False:
            return self.parse_sampleinformationtableline(line)
        else:
            return self.parse_quantitationesultsline(line)

    def parse_headerline(self, line):
        """ Parses header lines

            Header example:
            [Header]
            Data File Name\tC:\\GCMSsolution\\Data\\Pesticides\\OCt 2016\\CRACKER JACK_1-16-02258-001_10.qgd
            Output Date\t10/18/2016
            Output Time\t12:11:18 PM
            [File Information]
            Type\tData File
            Generated\t10/17/2016 4:25:40 PM
            Generated by\tAdmin
            Modified\t10/17/2016 4:43:02 PM'
            [Original Files]
            Data File\tC:\\GCMSsolution\\Data\\Pesticides\\OCt 2016\\CRACKER JACK_1-16-02258-001_10.qgd
            Method File\tC:\\GCMSsolution\\Data\\Pesticides\\OCt 2016\\Pesticide Method with IS June 13 2015.qgm
            Batch File\tC:\\GCMSsolution\\Data\\Pesticides\\OCt 2016\\20161017 samples.qgb
            Report Format File
            Tuning File\tC:\\GCMSsolution\\System\\Tune1\\Autotune09262016.qgt
        """
        if self._end_header == True:
            # Header already processed
            return 0

        if line.startswith(self.SAMPLEINFORMATIONTABLE_KEY):
            self._end_header = True
            if len(self._header) == 0:
                self.err("No header found", numline=self._numline)
                return -1
            return 0

        splitted = [token.strip() for token in line.split(self.SEPERATOR)]

        # [Header]
        if splitted[0] == self.HEADERTABLE_KEY:
            if self.HEADERTABLE_KEY in self._header:
                self.warn("Header [Header] Info already found. Discarding",
                          numline=self._numline, line=line)
                return 0

            self._header[self.HEADERTABLE_KEY] = []
            for i in range(len(splitted) - 1):
                if splitted[i + 1]:
                    self._header[self.HEADERTABLE_KEY].append(splitted[i + 1])

        # Data File Name, C:\GCMSsolution\Data\October\1-16-02249-001_CD_10172016_2.qgd
        elif splitted[0] == self.HEADERKEY_FILENAME:
            if self.HEADERKEY_FILENAME in self._header:
                self.warn("Header File Data Name already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.HEADERKEY_FILENAME] = splitted[1]
            else:
                self.warn("File Data Name not found or empty",
                          numline=self._numline, line=line)

        # Output Date	10/18/2016
        elif splitted[0] == self.HEADERKEY_OUTPUTDATE:
            if splitted[1]:
                try:
                    d = datetime.strptime(splitted[1], "%m/%d/%Y")
                    self._header[self.HEADERKEY_OUTPUTDATE] = d
                except ValueError:
                    self.err("Invalid Output Date format",
                             numline=self._numline, line=line)
            else:
                self.warn("Output Date not found or empty",
                          numline=self._numline, line=line)
                d = datetime.strptime(splitted[1], "%m/%d/%Y")

        # Output Time	12:04:11 PM
        elif splitted[0] == self.HEADERKEY_OUTPUTTIME:
            if splitted[1]:
                try:
                    d = datetime.strptime(splitted[1], "%I:%M:%S %p")
                    self._header[self.HEADERKEY_OUTPUTTIME] = d
                except ValueError:
                    self.err("Invalid Output Time format",
                             numline=self._numline, line=line)
            else:
                self.warn("Output Time not found or empty",
                          numline=self._numline, line=line)
                d = datetime.strptime(splitted[1], "%I:%M %p")

        # [File Information]
        if splitted[0] == self.FILEINFORMATION_KEY:
            if self.FILEINFORMATION_KEY in self._header:
                self.warn("Header [Header] Info already found. Discarding",
                          numline=self._numline, line=line)
                return 0

            self._header[self.FILEINFORMATION_KEY] = []
            for i in range(len(splitted) - 1):
                if splitted[i + 1]:
                    self._header[self.FILEINFORMATION_KEY].append(splitted[i + 1])

        # Type \t Data File
        elif splitted[0] == self.FILEINFORMATIONKEY_TYPE:
            if self.FILEINFORMATIONKEY_TYPE in self._header:
                self.warn("Header Type already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.FILEINFORMATIONKEY_TYPE] = splitted[1]
            else:
                self.warn("File Data Name not found or empty",
                          numline=self._numline, line=line)

        # Generated 10/17/2016 4:25:40 PM
        elif splitted[0] == self.FILEINFORMATIONKEY_GENERATED:
            if splitted[1]:
                try:
                    d = datetime.strptime(splitted[1], "%m/%d/%Y %I:%M:%S %p")
                    self._header[self.FILEINFORMATIONKEY_GENERATED] = d
                except ValueError:
                    self.err("Invalid Generated Date format",
                             numline=self._numline, line=line)
            else:
                self.warn("Generated Date not found or empty",
                          numline=self._numline, line=line)
                d = datetime.strptime(splitted[1], "%m/%d/%Y")

        # Generated by admin
        elif splitted[0] == self.FILEINFORMATIONKEY_GENERATEDBY:
            if self.FILEINFORMATIONKEY_GENERATEDBY in self._header:
                self.warn("Header Generated by already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.FILEINFORMATIONKEY_GENERATEDBY] = splitted[1]
            else:
                self.warn("File Data Name not found or empty",
                          numline=self._numline, line=line)

        # Modified 10/17/2016 4:25:40 PM
        elif splitted[0] == self.FILEINFORMATIONKEY_MODIFIED:
            if splitted[1]:
                try:
                    d = datetime.strptime(splitted[1], "%m/%d/%Y %I:%M:%S %p")
                    self._header[self.FILEINFORMATIONKEY_MODIFIED] = d
                except ValueError:
                    self.err("Invalid Modified Date format",
                             numline=self._numline, line=line)
            else:
                self.warn("Modified Date not found or empty",
                          numline=self._numline, line=line)
                d = datetime.strptime(splitted[1], "%m/%d/%Y")

        # Modified by admin
        elif splitted[0] == self.FILEINFORMATIONKEY_MODIFIEDBY:
            if self.FILEINFORMATIONKEY_MODIFIEDBY in self._header:
                self.warn("Header Modified by already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.FILEINFORMATIONKEY_MODIFIEDBY] = splitted[1]
            else:
                self.warn("File Data Name not found or empty",
                          numline=self._numline, line=line)

        # Data File\tC:\GCMSsolution\Data\Pesticides\OCt 2016\CRACKER JACK_1-16-02258-001_10.qgd
        elif splitted[0] == self.ORIGINALFILESKEY_DATAFILE:
            if self.ORIGINALFILESKEY_DATAFILE in self._header:
                self.warn("Header File Data already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.ORIGINALFILESKEY_DATAFILE] = splitted[1]
            else:
                self.warn("Data File not found or empty",
                          numline=self._numline, line=line)

        # Method File\tC:\GCMSsolution\Data\Pesticides\OCt 2016\Pesticide Method with IS June 13 2015.qgm
        elif splitted[0] == self.ORIGINALFILESKEY_METHODFILE:
            if self.ORIGINALFILESKEY_METHODFILE in self._header:
                self.warn("Header Method File already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.ORIGINALFILESKEY_METHODFILE] = splitted[1]
            else:
                self.warn("Method File not found or empty",
                          numline=self._numline, line=line)

        # Batch File\tC:\GCMSsolution\Data\Pesticides\OCt 2016\20161017 samples.qgb
        elif splitted[0] == self.ORIGINALFILESKEY_BATCHFILE:
            if self.ORIGINALFILESKEY_BATCHFILE in self._header:
                self.warn("Header Batch File already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.ORIGINALFILESKEY_BATCHFILE] = splitted[1]
            else:
                self.warn("Batch File not found or empty",
                          numline=self._numline, line=line)

        # Report Format File
        elif splitted[0] == self.ORIGINALFILESKEY_REPORTFORMATFILE:
            if self.ORIGINALFILESKEY_REPORTFORMATFILE in self._header:
                self.warn("Header Report Format File already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[0]:
                self._header[self.ORIGINALFILESKEY_REPORTFORMATFILE] = splitted[0]
            else:
                self.warn("Report Format File not found or empty",
                          numline=self._numline, line=line)

        # Tuning File\tC:\GCMSsolution\System\Tune1\Autotune09262016.qgt
        elif splitted[0] == self.ORIGINALFILESKEY_TUNINGFILE:
            if self.ORIGINALFILESKEY_TUNINGFILE in self._header:
                self.warn("Header Report Format File already found. Discarding",
                          numline=self._numline, line=line)
                return 0;

            if splitted[1]:
                self._header[self.ORIGINALFILESKEY_TUNINGFILE] = splitted[1]
            else:
                self.warn("Report Format File not found or empty",
                          numline=self._numline, line=line)

        # [File Description]
        elif line.startswith(self.FILEDESCRIPTION_KEY):
            self._end_header = True
            if len(self._header) == 0:
                self.err("No header found", numline=self._numline)
                return -1
            return 0

        if line.startswith(self.QUANTITATIONRESULTS_KEY):
            self._end_header = True
            if len(self._header) == 0:
                self.err("No header found", numline=self._numline)
                return -1
            return 0


        return 0

    def parse_sampleinformationtableline(self, line):
        """ Parses sample information table lines
            Sample Information Table example:
            [Sample Information]
            Operator Name	Admin
            Analyzed	10/17/2016 4:29:47 PM
            Type	Unknown
            Level	1
            Sample Name	CRACKER JACK
            Sample ID	1-16-02258-001
            ISTD Amount 1	1
            ISTD Amount 2	1
            ISTD Amount 3	1
            ISTD Amount 4	1
            ISTD Amount 5	1
            ISTD Amount 6	1
            ISTD Amount 7	1
            ISTD Amount 8	1
            ISTD Amount 9	1
            ISTD Amount 10	1
            ISTD Amount 11	1
            ISTD Amount 12	1
            ISTD Amount 13	1
            ISTD Amount 14	1
            ISTD Amount 15	1
            ISTD Amount 16	1
            ISTD Amount 17	1
            ISTD Amount 18	1
            ISTD Amount 19	1
            ISTD Amount 20	1
            ISTD Amount 21	1
            ISTD Amount 22	1
            ISTD Amount 23	1
            ISTD Amount 24	1
            ISTD Amount 25	1
            ISTD Amount 26	1
            ISTD Amount 27	1
            ISTD Amount 28	1
            ISTD Amount 29	1
            ISTD Amount 30	1
            ISTD Amount 31	1
            ISTD Amount 32	1
            Sample Amount	3.2397
            Dilution Factor	10
            Vial#	6
            Injection Volume	1
            Injection Count	1
            Bar Code
        """

        sampleinformationresult = {}
        # Operator Name \t Admin
        if line.startswith(self.SAMPLEINFORMATIONKEY_OPERATORNAME) \
                or line.startswith(self.SAMPLEINFORMATIONKEY_ANALYZED) \
                or line.startswith(self.SAMPLEINFORMATIONKEY_TYPE) \
                or line.startswith(self.SAMPLEINFORMATIONKEY_LEVEL) \
                or line.startswith(self.SAMPLEINFORMATIONKEY_SAMPLENAME) \
                or line.startswith(self.SAMPLEINFORMATIONKEY_SAMPLEID):
            # Nothing to do, continue
            return 0

        # Sample Amount\t 3.2397
        # Dilution factor\t10
        if line.startswith(self.SAMPLEINFORMATIONRESULT_DILUTIONFACTOR) \
                or line.startswith(self.SAMPLEINFORMATIONRESULT_SAMPLEAMOUNT): \

            splitted = [token.strip() for token in line.split(self.SEPERATOR)]
            sampleinformationresult[splitted[0]] =  float(splitted[1])
            self._sampleinformationresults.append(sampleinformationresult)
            return 0


        # [Original Files]
        # \t
        if line.startswith(self.ORIGINALFILESTABLE_KEY) \
            or line.startswith(self.SEPERATOR):
            self._end_sampleinformationtable = True
            self._end_header = False
            if len(self._sampleinformationresults) == 0:
                self.err("No Sample Information Table found", linenum=self._numline)
                return -1
            return 0


    def parse_quantitationesultsline(self, line):
        """ Parses quantitation result lines
            Please see samples/GC-QQQ output.txt
            [MS Quantitative Results] section
        """

        # [MS Quantitative Results]
        if line.startswith(self.QUANTITATIONRESULTS_KEY) \
                or line.startswith(self.QUANTITATIONRESULTS_NUMBEROFIDS) \
                or line.startswith(self.SIMILARITYSEARCHRESULTS_KEY) \
                or line.startswith(self.PEAK_TABLE_KEY): 

            # Nothing to do, continue
            return 0

        # # of IDs \t23
        if line.startswith(self.QUANTITATIONRESULTS_HEADER_ID_NUMBER):
            self._quantitationresultsheader = [token.strip() for token in line.split('\t') if token.strip()]
            return 0

        #1 \talpha-Pinene \tTarget \t0 \t93.00 \t7.738 \t7.680 \t7.795 \t2.480
        #\t344488 \t138926 \t0.02604 \tAuto \t2	\t7.812	\tLinear \t0 \t0
        #\t4.44061e+008	\t278569 \t0 \t0 \t38.94 \t38.58 \t0.00	\t98 \t92.00
        #\t0 \t0 \t38.94 \t38.58 \t91.00 \t0 \t0 \t38.93 \t40.02 \t0 \t0 \t0
        #\t0 \t0 \t0 \t0 #\t0 \t0 \t0 \t0 \t0 \t0 \t0 \t0 \t75.27 \tmg \t0.00000
        splitted = [token.strip() for token in line.split('\t')]
        quantitation = {'DefaultResult': 'Conc.'}
        if len(self._sampleinformationresults) == 2:
            sample_amount = self._sampleinformationresults[0]
            dilution_factor = self._sampleinformationresults[1]
            quantitation['Sample Amount'] = sample_amount['Sample Amount']
            quantitation['Dilution Factor'] = dilution_factor['Dilution Factor']

        for colname in self._quantitationresultsheader:
            quantitation[colname] = ''

        for i in range(len(splitted)):
            token = splitted[i]
            if i < len(self._quantitationresultsheader):
                colname = self._quantitationresultsheader[i]
                if colname in self.QUANTITATIONRESULTS_NUMERICHEADERS:
                    try:
                        quantitation[colname] = float(token)
                    except ValueError:
                        self.warn(
                            "No valid number ${token} in column ${index} (${column_name})",
                            mapping={"token": token,
                                     "index": str(i + 1),
                                     "column_name": colname},
                            numline=self._numline, line=line)
                        quantitation[colname] = token
                else:
                    quantitation[colname] = token

                val = re.sub(r"\W", "", splitted[1])
                self._addRawResult(quantitation['ID#'],
                                   values={val:quantitation},
                                   override=True)
            elif token:
                self.err("Orphan value in column ${index} (${token})",
                         mapping={"index": str(i+1),
                                  "token": token},
                         numline=self._numline, line=line)


class GCMSQP2010SEImporter(AnalysisResultsImporter):

    def __init__(self, parser, context, idsearchcriteria, override,
                 allowed_ar_states=None, allowed_analysis_states=None,
                 instrument_uid=''):
        AnalysisResultsImporter.__init__(self, parser, context, idsearchcriteria,
                                         override, allowed_ar_states,
                                         allowed_analysis_states,
                                         instrument_uid)
